library(data.table)
library(dplyr)
library(reshape2)
library(sampling)
library(caret)
library(party)
library(pROC)
library(e1071)


#------------技能大赛------------------------------------------
dt.update<-fread("F:\\ZHZ_R\\RFVALL\\RFV整合201803-201808_180917updated.csv"
                 ,integer64 = 'numeric')

setwd("F:\\ZHZ_R\\LOST\\V2\\")

dt.5.pro<-fread("大数据大赛7月掌厅yly_20180814_103213_223510.csv",
                integer64 = 'numeric')
dt.6.pro<-fread("8月份掌厅活跃用户信息-rzl_20180913_170913_227586.csv",
                integer64 = 'numeric')
#
# [1] "mobile"                   "ONLINE_ID"                "AGE"                     
# [4] "CROWD_MAIN_PHONE_NO_FLAG" "GRP_TOGETHER_FLAG"        "DZD_TYPE"                
# [7] "GPRS_MISS_FLOW"           "BFR_ALCT_TOTAL_FEE"   
setnames(dt.6.pro,6:8,c("DZD_TYPE","BFR_ALCT_TOTAL_FEE","GPRS_MISS_FLOW"))
dt.5.pv<-fread("RFV_V_按pv汇总_201807.csv",
               integer64 = 'numeric')
dt.6.pv<-fread("RFV_V_按pv汇总_201808.csv",
               integer64 = 'numeric')

dt.5.rfv<-fread("RFV_FV全量清单(含预测)_201807.csv",
                integer64 = 'numeric')
dt.6.rfv<-fread("RFV_FV全量清单_201808.csv",
                integer64 = 'numeric')

# "分享"   "业务"   "签到"  "礼包"   "查询"   "充值"   "登录设置等其他服务"
# [9] "底部切换"   "关闭弹框"           "自动"   "搜索"  
# [13] "活动"   "数字化内容"         "qita"     
newNames<-c("FX",'YW','QD','LB','CX','CZ','DL','DH','GB','ZD','SS','HD','SZ','QT')
setnames(dt.5.pv,2:15,newNames)
setnames(dt.6.pv,2:15,newNames)
dt.5.pv<-dt.5.pv[,.(mobile,FX,YW,QD,LB,CX,CZ,SS,HD,SZ)]
dt.6.pv<-dt.6.pv[,.(mobile,FX,YW,QD,LB,CX,CZ,SS,HD,SZ)]
pp<-function(dt,dt.up){
  
  setnames(dt,1,'mobile')
  dt.up<-dt.up[,.(mobile,rfv_f,vv_sum,vv_avg,shang,zhong,xia)]
  dt.up<-dt.up[dt.up$mobile %in% dt$mobile]
  dt.up<-dt[dt.up,on='mobile']
  #缺失处理
  dt.up$GRP_TOGETHER_FLAG[is.na(dt.up$GRP_TOGETHER_FLAG)]<-0
  dt.up$CROWD_MAIN_PHONE_NO_FLAG[is.na(dt.up$CROWD_MAIN_PHONE_NO_FLAG)]<-0
  dt.up$DZD_TYPE[is.na(dt.up$DZD_TYPE)]<-0
  for(i in "AGE")
    set(dt.up,i = which(is.na(dt.up[[i]])),j=i,value = median(dt.up$AGE,na.rm = T))
  for(i in "ONLINE_ID")
    set(dt.up,i = which(is.na(dt.up[[i]])),j=i,value = median(dt.up$ONLINE_ID,na.rm = T))
  for(i in "GPRS_MISS_FLOW")
    set(dt.up,i = which(is.na(dt.up[[i]])),j=i,value = median(dt.up$GPRS_MISS_FLOW,na.rm = T))
  for(i in "BFR_ALCT_TOTAL_FEE")
    set(dt.up,i = which(is.na(dt.up[[i]])),j=i,value = median(dt.up$BFR_ALCT_TOTAL_FEE,na.rm = T))
  dt.up<-dt.up[DZD_TYPE==1,DZD_TYPE:=-1]
  dt.up<-dt.up[DZD_TYPE==2,DZD_TYPE:=1]
  
  return(dt.up)
}

dt5<-pp(dt.5.pro,dt.5.rfv)
dt6<-pp(dt.6.pro,dt.6.rfv)

dt5<-dt.5.pv[dt5,on='mobile']
dt5<-dt5[!is.na(dt5$FX)]

dt6<-dt.6.pv[dt6,on='mobile']
dt6<-dt6[!is.na(dt6$FX)]

rm(dt.5.pro,dt.6.pro,
   dt.5.rfv,dt.6.rfv,
   dt.5.pv,dt.6.pv)


for(i in 2:length(names(dt5))){
  names(dt5)[i]<-paste(names(dt5)[i],"_l",sep="")
}


dt.delta<-dt5[dt6,on='mobile']
dt.delta<-dt.delta[!is.na(dt.delta$FX_l)]

dt.delta2<-dt.delta[,.(
  mobile,
  FX_d=FX_l-FX,
  YW_d=YW_l-YW,
  QD_d=QD_l-QD,
  LB_d=LB_l-LB,
  CX_d=CX_l-CX,
  CZ_d=CZ_l-CZ,
  SS_d=SS_l-SS,
  HD_d=HD_l-HD,
  SZ_d=SZ_l-SZ,
  GPRS_MISS_FLOW_d=GPRS_MISS_FLOW_l-GPRS_MISS_FLOW,
  BFR_ALCT_TOTAL_FEE_d=BFR_ALCT_TOTAL_FEE_l-BFR_ALCT_TOTAL_FEE,
  rfv_f_d=rfv_f_l-rfv_f,
  vv_sum_d=vv_sum_l-vv_sum,
  vv_avg_d=vv_avg_l-vv_avg,
  shang_d=shang_l-shang,
  zhong_d=zhong_l-zhong,
  xia_d=xia_l-xia
)]

colSums(is.na(dt6))
dt.6.allsample<-dt.delta2[dt6,on='mobile']
dt.7lost<-dt.update[,.(mobile,lost201809)]#修改列名称
#dt.6.allsample$mobile<-as.character(dt.6.allsample$mobile)
setnames(dt.7lost,2,"lost")
dt.6.allsample<-dt.7lost[dt.6.allsample,on='mobile']

colSums(is.na(dt.6.allsample))

#有两个月数据
dt.6.allsample.2m<-dt.6.allsample[!is.na(FX_d)]
mean(dt.6.allsample.2m$lost)

#新客
dt.6.allsample.1m<-dt.6.allsample[is.na(FX_d)]
mean(dt.6.allsample.1m$lost)

rm(dt.7lost,dt.delta,dt.delta2,dt.update,dt5,dt6)

dt.2m<-dt.6.allsample.2m
dt.1m<-dt.6.allsample.1m

rm(dt.6.allsample.2m,dt.6.allsample.1m,dt.6.allsample)
mean(dt.2m$lost)
mean(dt.1m$lost)

#抽样2m
train.id.2m<-strata(dt.2m,stratanames = "lost",
                 size = c(1335000,165000),description=T)#存留、流失
train.dt.2m<-getdata(dt.2m,train.id.2m)
train.dt.2m<-train.dt.2m[,Prob:=NULL]
train.dt.2m<-train.dt.2m[,ID_unit:=NULL]
train.dt.2m<-train.dt.2m[,Stratum:=NULL]

#抽取测试集
test.dt.2m<-dt.2m[!dt.2m$mobile %in% train.dt.2m$mobile]

#抽样2mlite
train.id.2mlite<-strata(dt.2m,stratanames = "lost",
                    size = c(26700,3300),description=T)#存留、流失
train.dt.2mlite<-getdata(dt.2m,train.id.2mlite)
train.dt.2mlite<-train.dt.2mlite[,Prob:=NULL]
train.dt.2mlite<-train.dt.2mlite[,ID_unit:=NULL]
train.dt.2mlite<-train.dt.2mlite[,Stratum:=NULL]




#抽样1m
train.id.1m<-strata(dt.1m,stratanames = "lost",
                    size = c(290000,210000),description=T)#存留、流失
train.dt.1m<-getdata(dt.1m,train.id.1m)
train.dt.1m<-train.dt.1m[,Prob:=NULL]
train.dt.1m<-train.dt.1m[,ID_unit:=NULL]
train.dt.1m<-train.dt.1m[,Stratum:=NULL]


#抽取测试集
test.dt.1m<-dt.1m[!dt.1m$mobile %in% train.dt.1m$mobile]
#抽样1mlite
train.id.1mlite<-strata(dt.1m,stratanames = "lost",
                        size = c(14500,10500),description=T)#存留、流失
train.dt.1mlite<-getdata(dt.1m,train.id.1mlite)
train.dt.1mlite<-train.dt.1mlite[,Prob:=NULL]
train.dt.1mlite<-train.dt.1mlite[,ID_unit:=NULL]
train.dt.1mlite<-train.dt.1mlite[,Stratum:=NULL]




#导出固定训练和测试集
fwrite(train.dt.1m,"train_dt_1m.csv")
fwrite(train.dt.2m,"train_dt_2m.csv")
fwrite(test.dt.1m,"test_dt_1m.csv")
fwrite(test.dt.2m,"test_dt_2m.csv")
fwrite(train.dt.1mlite,"train_dt_1mlite.csv")
fwrite(train.dt.2mlite,"train_dt_2mlite.csv")

rm(train.id.1m,train.id.2m,
   train.id.1mlite,train.id.2mlite)

#-------训练 与 测试抽样完成 ------------------------------------------
# #RF重要性
# library(randomForest)
# #2m
# rf.train.2m<-randomForest(as.factor(lost201807) ~ .,
#                        data=train.dt.2mlite[,c(2:41)],importance = T)
# importance(rf.train.2m)
# rf.train.imp.2m<-as.data.frame(rf.train.2m$importance)
# rf.train.imp.2m$pmt<-row.names(rf.train.imp.2m)
# rf.train.imp.2m<-as.data.table(rf.train.imp.2m)
# fwrite(rf.train.imp.2m,'rf_train_imp_2m.csv')
# 
# #1m
# rf.train.1m<-randomForest(as.factor(lost201807) ~ .,
#                           data=train.dt.1mlite[,c(19:41)],importance = T)
# importance(rf.train.1m)
# rf.train.imp.1m<-as.data.frame(rf.train.1m$importance)
# rf.train.imp.1m$pmt<-row.names(rf.train.imp.1m)
# rf.train.imp.1m<-as.data.table(rf.train.imp.1m)
# fwrite(rf.train.imp.1m,'rf_train_imp_1m.csv')
# 
# #
# #相关性
# library('psych')
# library('reshape2')
# #m2
# train.dt.cor<-corr.test(train.dt.2m[,c(2:40)])
# 
# a<-as.data.frame(train.dt.cor$r)
# a2<-as.data.frame(train.dt.cor$p)
# a$pmt<-row.names(a)
# a2$pmt<-row.names(a2)
# a3<-melt(a,id.vars = c('pmt'),measure.vars = row.names(a))
# a3.p<-melt(a2,id.vars = c('pmt'),measure.vars = row.names(a2))
# a3<-a3[a3$pmt!=a3$variable,]
# a3.p<-a3.p[a3.p$pmt!=a3.p$variable,]
# a3<-as.data.table(a3)
# a3.p<-as.data.table(a3.p)
# 
# train.dt.2m.cor.re<-a3.p[a3,on=c('pmt','variable')]
# train.dt.2m.cor.re.high<-train.dt.2m.cor.re[(i.value>=0.7 | i.value<=-0.7)& value<=0.05]
# rm(a,a2,a3,a3.p,train.dt.cor)
# 
# train.dt.2m.cor.re.high$variable<-as.character(train.dt.2m.cor.re.high$variable)
# cor.g1<-c('QD_d',train.dt.2m.cor.re.high[(pmt==p.win1.2m[1]),variable],p.win1.2m[1])
# cor.g2<-c(train.dt.2m.cor.re.high[(pmt==p.win1.2m[2]),variable],p.win1.2m[2])
# cor.g3<-c(train.dt.2m.cor.re.high[(pmt==p.win1.2m[3]),variable],p.win1.2m[3])
# cor.g4<-c(train.dt.2m.cor.re.high[(pmt==p.win1.2m[4]),variable],p.win1.2m[4])
# cor.g5<-c(train.dt.2m.cor.re.high[(pmt==p.win1.2m[5]),variable],p.win1.2m[5])
# 
# library(PerformanceAnalytics)#加载包
# chart.Correlation(as.data.frame(train.dt.2mlite)[,c(cor.g2)], histogram=TRUE)
# 
# 
# 
# 
# #m1
# train.dt.cor<-corr.test(train.dt.1m[,c(19:40)])
# 
# a<-as.data.frame(train.dt.cor$r)
# a2<-as.data.frame(train.dt.cor$p)
# a$pmt<-row.names(a)
# a2$pmt<-row.names(a2)
# a3<-melt(a,id.vars = c('pmt'),measure.vars = row.names(a))
# a3.p<-melt(a2,id.vars = c('pmt'),measure.vars = row.names(a2))
# a3<-a3[a3$pmt!=a3$variable,]
# a3.p<-a3.p[a3.p$pmt!=a3.p$variable,]
# a3<-as.data.table(a3)
# a3.p<-as.data.table(a3.p)
# 
# train.dt.1m.cor.re<-a3.p[a3,on=c('pmt','variable')]
# train.dt.1m.cor.re.high<-train.dt.1m.cor.re[(i.value>=0.7 | i.value<=-0.7)& value<=0.05]
# rm(a,a2,a3,a3.p,train.dt.cor)
# 
# #高相关对比筛选
# #2m
# p.a<-train.dt.2m.cor.re.high$pmt
# p.b<-as.character(train.dt.2m.cor.re.high$variable)
# pmt.cor<-unique(c(p.a,p.b))
# pmt.UNcor<-names(train.dt.2m[,c(2:40)])[!names(train.dt.2m[,c(2:40)]) %in% pmt.cor]
# p.fail<-c()
# for (i  in 1:length(p.a)) {
#   if(rf.train.imp.2m[pmt==p.a[i],MeanDecreaseGini] >=rf.train.imp.2m[pmt==p.b[i],MeanDecreaseGini]){
#     p.fail[i]<-p.b[i]
#   }else{
#     p.fail[i]<-p.a[i]
#   }
# }
# p.fail<-unique(p.fail)
# p.win1.2m<-pmt.cor[!pmt.cor %in% p.fail]
# p.final.2m<-unique(c(pmt.UNcor,p.win1.2m))
# 
# #1m
# p.a<-train.dt.1m.cor.re.high$pmt
# p.b<-as.character(train.dt.1m.cor.re.high$variable)
# pmt.cor<-unique(c(p.a,p.b))
# pmt.UNcor<-names(train.dt.1m[,c(19:40)])[!names(train.dt.1m[,c(19:40)]) %in% pmt.cor]
# p.fail<-c()
# for (i  in 1:length(p.a)) {
#   if(rf.train.imp.1m[pmt==p.a[i],MeanDecreaseGini] >=rf.train.imp.1m[pmt==p.b[i],MeanDecreaseGini]){
#     p.fail[i]<-p.b[i]
#   }else{
#     p.fail[i]<-p.a[i]
#   }
# }
# p.fail<-unique(p.fail)
# p.win1.1m<-pmt.cor[!pmt.cor %in% p.fail]
# p.final.1m<-unique(c(pmt.UNcor,p.win1.1m))
# 
# rm(p.a,p.b,pmt.cor,pmt.UNcor,p.fail)
# 
# library(PerformanceAnalytics)#加载包
# chart.Correlation(as.data.frame(train.dt.1mlite)[,c(pmt.cor)], histogram=TRUE)
# 




#去除相关性后剩余的变量
# print(paste(c(p.final.2m),collapse = "','"))
p.final.2m<-c('FX_d','YW_d','LB_d','CX_d','CZ_d',
              'GPRS_MISS_FLOW_d','BFR_ALCT_TOTAL_FEE_d',
              'vv_avg_d','xia_d','FX','YW','CX','CZ','ONLINE_ID',
              'AGE','CROWD_MAIN_PHONE_NO_FLAG','GRP_TOGETHER_FLAG',
              'DZD_TYPE','GPRS_MISS_FLOW','BFR_ALCT_TOTAL_FEE',
              'vv_avg','vv_sum_d','vv_sum','SS_d','HD_d','SZ_d')
# print(paste(c(p.final.1m),collapse = "','"))
p.final.1m<-c('FX','YW','CX','CZ','SS','HD','SZ',
              'ONLINE_ID','AGE','CROWD_MAIN_PHONE_NO_FLAG','GRP_TOGETHER_FLAG',
              'DZD_TYPE','GPRS_MISS_FLOW','BFR_ALCT_TOTAL_FEE',
              'vv_avg','shang','rfv_f')


#模型系数
#2mm
# model_LR.2m<-glm(as.factor(lost) ~ .,family=binomial(link='logit'),
#               data=as.data.frame(train.dt.2m)[,c(p.final.2m,'lost')])
# summary(model_LR.2m)

p.final.2m.upd<-p.final.2m[!p.final.2m %in% c('SZ_d','FX')]
#p.final.2m.upd<-tobetest[-c(1:16)]

model_LR.2m<-glm(as.factor(lost) ~ .,family=binomial(link='logit'),
                 data=as.data.frame(train.dt.2m)[,c(p.final.2m.upd,'lost')])
summary(model_LR.2m)

# model_LR.2m.step<-stepAIC(model_LR.2m,direction = 'backward',trace = F)
# #summary(model_LR.2m.step)
# model_LR.2m.step$anova



pre<-predict(model_LR.2m,type='response')
r_c<-roc(train.dt.2m$lost, pre, plot=TRUE, print.thres=TRUE, print.auc=TRUE)  
r_c$auc
c.pred<-predict(model_LR.2m,test.dt.2m,type='response')

#--阈值2--
v2m<-0.13
c.pred <- ifelse( c.pred > v2m,1,0)
a<-t(table(c.pred,test.dt.2m$lost))
LR_recall<-a[2,2]/(a[2,1]+a[2,2])
LR_precision<-a[2,2]/(a[1,2]+a[2,2])
LR_F1<-2*LR_recall*LR_precision/(LR_recall+LR_precision)
LR_F2<-5*LR_recall*LR_precision/(LR_recall+4*LR_precision)

print(a)
print(sum(c.pred))
print(LR_recall)
print(LR_precision)
print(LR_F1)
print(LR_F2)


# #
# library(ggplot2)
# test.dt.2m$p<-c.pred
# quantile(test.dt.2m$vv_sum[test.dt.2m$p==0 &test.dt.2m$lost==0],c(0.05,0.1,0.5,0.8,0.9,0.95))
# quantile(test.dt.2m$vv_sum[test.dt.2m$p==1 &test.dt.2m$lost==1],c(0.05,0.1,0.5,0.8,0.9,0.95))
# 
# quantile(test.dt.2m$vv_sum_d[test.dt.2m$p==0 &test.dt.2m$lost==0],c(0.05,0.1,0.5,0.8,0.9,0.95))
# quantile(test.dt.2m$vv_sum_d[test.dt.2m$p==1 &test.dt.2m$lost==1],c(0.05,0.1,0.5,0.8,0.9,0.95))
# 
# 
# quantile(test.dt.2m$xia_d[test.dt.2m$p==0 &test.dt.2m$lost==0],c(0.05,0.1,0.5,0.8,0.9,0.95))
# quantile(test.dt.2m$xia_d[test.dt.2m$p==1 &test.dt.2m$lost==1],c(0.05,0.1,0.5,0.8,0.9,0.95))
# 
# quantile(test.dt.2m$BFR_ALCT_TOTAL_FEE_d[test.dt.2m$p==0 &test.dt.2m$lost==0],c(0.05,0.1,0.5,0.8,0.9,0.95))
# quantile(test.dt.2m$BFR_ALCT_TOTAL_FEE_d[test.dt.2m$p==1 &test.dt.2m$lost==1],c(0.05,0.1,0.5,0.8,0.9,0.95))
# 
# test.dt.2m$p2<-as.factor(test.dt.2m$p)
# ggplot(test.dt.2m, aes(vv_sum, color = p2)) +
#   geom_density(bin = 2)+xlim(0, 10)
# 
# library(rpart)
# tree.both<-rpart(as.factor(lost)~ .,data=as.data.frame(train.dt.2m)[,c(p.final.2m.upd,'lost')],
#                  method='class',minsplit=20,minbucket=150,cp=0.00017)
# summary(tree.both)
# tree.both2<-prune(tree.both,cp=tree.both$cptable[which.min(tree.both$cptable[,"xerror"]),"CP"])
# summary(tree.both2)
# 

# #loop
# rf.train.imp.2m <- rf.train.imp.2m[order(rf.train.imp.2m$MeanDecreaseGini), ]  # sort
# tobetest<-rf.train.imp.2m$pmt
# tobetest<-tobetest[tobetest %in% p.final.2m.upd]
# 
# p.final2<-p.final.2m.upd
# para_test_LR<-c()
# para_test_LR_recall<-c()
# para_test_LR_precision<-c()
# para_test_LR_F1score<-c()
# para_test_LR_F2score<-c()
# x=1
# for(i in 1:(length(tobetest)-3)){
#   p.final2<-tobetest[-c(1:i)]
#   print(i)
#   model_LR<-glm(as.factor(lost201807) ~ .,family=binomial(link='logit'),
#                 data=as.data.frame(train.dt.2m)[,c(p.final2,'lost201807')])
#   pre<-predict(model_LR,type='response')
#   r_c<-roc(train.dt.2m$lost201807, pre, plot=FALSE)
#   print(r_c$auc)
#   # #预测 all
#   c.pred<-predict(model_LR,test.dt.2m,type='response')
#   #r_c<-roc(test.dt$lost_5, c.pred, plot=TRUE, print.thres=TRUE, print.auc=TRUE)
#   c.pred <- ifelse( c.pred > 0.129,1,0)
#   #print(table(c.pred))
#   a<-t(table(c.pred,test.dt.2m$lost201807))
#   #print(a)
#   para_test_LR[x]<-paste(c(p.final2),collapse = "_")
#   #print(para_test_LR[x])
#   para_test_LR_recall[x]<-a[2,2]/(a[2,1]+a[2,2])
#   print(para_test_LR_recall[x])
#   para_test_LR_precision[x]<-a[2,2]/(a[1,2]+a[2,2])
#   print(para_test_LR_precision[x])
#   para_test_LR_F1score<-(para_test_LR_recall[x]*para_test_LR_precision[x])/(para_test_LR_recall[x]+para_test_LR_precision[x])
#   para_test_LR_F2score<-(5*para_test_LR_recall[x]*para_test_LR_precision[x])/(para_test_LR_recall[x]+4*para_test_LR_precision[x])
#   x=x+1
# }  
# testReport.LR<-data.frame(
#   zuhe=para_test_LR,
#   precision=para_test_LR_precision,
#   recall=para_test_LR_recall,
#   f1=para_test_LR_F1score,
#   f2=para_test_LR_F2score
# )



#1m
# model_LR.1m<-glm(as.factor(lost201807) ~ .,family=binomial(link='logit'),
#                  data=as.data.frame(train.dt.1m)[,c(p.final.1m,'lost201807')])
# summary(model_LR.1m)

p.final.1m.upd<-p.final.1m[!p.final.1m %in% c('SZ','FX')]
model_LR.1m<-glm(as.factor(lost) ~ .,family=binomial(link='logit'),
                 data=as.data.frame(train.dt.1m)[,c(p.final.1m.upd,'lost')])
summary(model_LR.1m)
# anova(model_LR.1m, test = 'Chisq')
# library(MASS)
# model_LR.1m.step<-stepAIC(model_LR.1m,direction = 'backward',trace = F)
# summary(model_LR.1m.step)
# model_LR.1m.step$anova
# 
# model_LR.1m.fw<-glm(as.factor(lost201807) ~ rfv_f,family=binomial(link='logit'),
#                     data=as.data.frame(train.dt.1m)[,c(p.final.1m.upd,'lost201807')])
# summary(model_LR.1m.fw)
# model_LR.1m.fw2<-stepAIC(model_LR.1m.fw,direction = 'both')
# model_LR.1m.fw2$anova
# 
# #1mXY图
# 
# getplot<-function(v,low,up,stp){
#   df<-data.frame(
#     cut=cut(v, breaks = seq(low, up, by = stp)),
#     lost=train.dt.1m$lost201807
#   )
#   df<-df %>% group_by(cut) %>% summarise(LR=mean(lost),n=n())
# }
# summary(train.dt.2m$CX_d)
# b<-getplot(train.dt.2m$CX_d,-10,10,5)


# a<-smbinning(train.dt.1m, y='lost201807', x='CX', p = 0.05)
# a.df<-data.frame(
#   cut=a$ivtable$Cutpoint,
#   lostr=a$ivtable$GoodRate,
#   counts=a$ivtable$CntRec
# )
# fwrite(b,'plot_2m_vv_sum.csv')


pre<-predict(model_LR.1m,type='response')
r_c<-roc(train.dt.1m$lost, pre, plot=TRUE, print.thres=TRUE, print.auc=TRUE)  
r_c$auc
c.pred<-predict(model_LR.1m,test.dt.1m,type='response')

#--阈值--
v1m<-0.408
c.pred <- ifelse( c.pred > v1m,1,0)#调低一些，拉高查全
a<-t(table(c.pred,test.dt.1m$lost))
LR_recall<-a[2,2]/(a[2,1]+a[2,2])
LR_precision<-a[2,2]/(a[1,2]+a[2,2])
LR_F1<-2*LR_recall*LR_precision/(LR_recall+LR_precision)
LR_F2<-5*LR_recall*LR_precision/(LR_recall+4*LR_precision)

print(a)
print(LR_recall)
print(LR_precision)
# print(LR_F1)
# print(LR_F2)

#loop
# rf.train.imp.1m <- rf.train.imp.1m[order(rf.train.imp.1m$MeanDecreaseGini), ]  # sort
# tobetest<-rf.train.imp.1m$pmt
# tobetest<-tobetest[tobetest %in% p.final.1m.upd]
# 
# p.final2<-p.final.1m.upd
# para_test_LR<-c()
# para_test_LR_recall<-c()
# para_test_LR_precision<-c()
# para_test_LR_F1score<-c()
# para_test_LR_F2score<-c()
# x=1
# for(i in 1:(length(tobetest)-3)){
#   p.final2<-tobetest[-c(1:i)]
#   print(i)
#   model_LR<-glm(as.factor(lost201807) ~ .,family=binomial(link='logit'),
#                 data=as.data.frame(train.dt.1m)[,c(p.final2,'lost201807')])
#   pre<-predict(model_LR,type='response')
#   r_c<-roc(train.dt.1m$lost201807, pre, plot=FALSE)
#   print(r_c$auc)
#   # #预测 all
#   c.pred<-predict(model_LR,test.dt.1m,type='response')
#   #r_c<-roc(test.dt$lost_5, c.pred, plot=TRUE, print.thres=TRUE, print.auc=TRUE)
#   c.pred <- ifelse( c.pred > 0.434,1,0)
#   #print(table(c.pred))
#   a<-t(table(c.pred,test.dt.1m$lost201807))
#   #print(a)
#   para_test_LR[x]<-paste(c(p.final2),collapse = "_")
#   #print(para_test_LR[x])
#   para_test_LR_recall[x]<-a[2,2]/(a[2,1]+a[2,2])
#   print(para_test_LR_recall[x])
#   para_test_LR_precision[x]<-a[2,2]/(a[1,2]+a[2,2])
#   print(para_test_LR_precision[x])
#   para_test_LR_F1score<-(para_test_LR_recall[x]*para_test_LR_precision[x])/(para_test_LR_recall[x]+para_test_LR_precision[x])
#   para_test_LR_F2score<-(5*para_test_LR_recall[x]*para_test_LR_precision[x])/(para_test_LR_recall[x]+4*para_test_LR_precision[x])
#   x=x+1
# }  
# testReport.LR.1m<-data.frame(
#   zuhe=para_test_LR,
#   precision=para_test_LR_precision,
#   recall=para_test_LR_recall,
#   f1=para_test_LR_F1score,
#   f2=para_test_LR_F2score
# )
# 

rm(dt.1m,dt.2m,r_c,test.dt.1m,test.dt.2m,
   train.dt.1m,train.dt.1mlite,
   train.dt.2m,train.dt.2mlite,
   train.id.1m,train.id.1mlite,
   train.id.2m,train.id.2mlite)


#---------6&7预测7在8流失-------------
# dt.update<-fread("F:\\ZHZ_R\\RFVALL\\RFV整合201803-201807_180904updated.csv"
#                  ,integer64 = 'numeric')
setwd("F:\\ZHZ_R\\LOST\\V2\\")
dt.5.pro<-fread("8月份掌厅活跃用户信息-rzl_20180913_170913_227586.csv",
                integer64 = 'numeric')
dt.6.pro<-fread("9月份掌厅活跃用户信息-rzl_20181009_134920_229893.csv",
                integer64 = 'numeric')

dt.5.pv<-fread("RFV_V_按pv汇总_201808.csv",
               integer64 = 'numeric')
dt.6.pv<-fread("RFV_V_按pv汇总_201809.csv",
               integer64 = 'numeric')

dt.5.rfv<-fread("RFV_FV全量清单_201808.csv",
                integer64 = 'numeric')
dt.6.rfv<-fread("RFV_FV全量清单_201809.csv",
                integer64 = 'numeric')

newNames<-c("FX",'YW','QD','LB','CX','CZ','DL','DH','GB','ZD','SS','HD','SZ','QT')
setnames(dt.5.pv,2:15,newNames)
setnames(dt.6.pv,2:15,newNames)
dt.5.pv<-dt.5.pv[,.(mobile,FX,YW,QD,LB,CX,CZ,SS,HD,SZ)]
dt.6.pv<-dt.6.pv[,.(mobile,FX,YW,QD,LB,CX,CZ,SS,HD,SZ)]


setnames(dt.6.pro,6:8,c(
  'DZD_TYPE','BFR_ALCT_TOTAL_FEE','GPRS_MISS_FLOW'
))
setnames(dt.5.pro,6:8,c(
  'DZD_TYPE','BFR_ALCT_TOTAL_FEE','GPRS_MISS_FLOW'
))

pp<-function(dt,dt.up){
  setnames(dt,1,'mobile')
  dt.up<-dt.up[,.(mobile,rfv_f,vv_sum,vv_avg,shang,zhong,xia)]
  dt.up<-dt.up[dt.up$mobile %in% dt$mobile]
  dt.up<-dt[dt.up,on='mobile']
  #缺失处理
  dt.up$GRP_TOGETHER_FLAG[is.na(dt.up$GRP_TOGETHER_FLAG)]<-0
  dt.up$CROWD_MAIN_PHONE_NO_FLAG[is.na(dt.up$CROWD_MAIN_PHONE_NO_FLAG)]<-0
  dt.up$DZD_TYPE[is.na(dt.up$DZD_TYPE)]<-0
  for(i in "AGE")
    set(dt.up,i = which(is.na(dt.up[[i]])),j=i,value = median(dt.up$AGE,na.rm = T))
  for(i in "ONLINE_ID")
    set(dt.up,i = which(is.na(dt.up[[i]])),j=i,value = median(dt.up$ONLINE_ID,na.rm = T))
  for(i in "GPRS_MISS_FLOW")
    set(dt.up,i = which(is.na(dt.up[[i]])),j=i,value = median(dt.up$GPRS_MISS_FLOW,na.rm = T))
  for(i in "BFR_ALCT_TOTAL_FEE")
    set(dt.up,i = which(is.na(dt.up[[i]])),j=i,value = median(dt.up$BFR_ALCT_TOTAL_FEE,na.rm = T))
  dt.up<-dt.up[DZD_TYPE==1,DZD_TYPE:=-1]
  dt.up<-dt.up[DZD_TYPE==2,DZD_TYPE:=1]
  
  return(dt.up)
}

dt5<-pp(dt.5.pro,dt.5.rfv)
dt6<-pp(dt.6.pro,dt.6.rfv)

dt5<-dt.5.pv[dt5,on='mobile']
dt5<-dt5[!is.na(dt5$FX)]

dt6<-dt.6.pv[dt6,on='mobile']
dt6<-dt6[!is.na(dt6$FX)]

rm(dt.5.pro,dt.6.pro,
   dt.5.rfv,dt.6.rfv,
   dt.5.pv,dt.6.pv)


for(i in 2:length(names(dt5))){
  names(dt5)[i]<-paste(names(dt5)[i],"_l",sep="")
}


dt.delta<-dt5[dt6,on='mobile']
dt.delta<-dt.delta[!is.na(dt.delta$FX_l)]

dt.delta2<-dt.delta[,.(
  mobile,
  FX_d=FX_l-FX,
  YW_d=YW_l-YW,
  QD_d=QD_l-QD,
  LB_d=LB_l-LB,
  CX_d=CX_l-CX,
  CZ_d=CZ_l-CZ,
  SS_d=SS_l-SS,
  HD_d=HD_l-HD,
  SZ_d=SZ_l-SZ,
  GPRS_MISS_FLOW_d=GPRS_MISS_FLOW_l-GPRS_MISS_FLOW,
  BFR_ALCT_TOTAL_FEE_d=BFR_ALCT_TOTAL_FEE_l-BFR_ALCT_TOTAL_FEE,
  rfv_f_d=rfv_f_l-rfv_f,
  vv_sum_d=vv_sum_l-vv_sum,
  vv_avg_d=vv_avg_l-vv_avg,
  shang_d=shang_l-shang,
  zhong_d=zhong_l-zhong,
  xia_d=xia_l-xia
)]

dt.6.allsample<-dt.delta2[dt6,on='mobile']

#有两个月数据
dt.6.allsample.2m<-dt.6.allsample[!is.na(FX_d)]

#新客
dt.6.allsample.1m<-dt.6.allsample[is.na(FX_d)]
#mean(dt.6.allsample.1m$lost201807)

rm(dt.delta,dt.delta2,dt.update,dt5,dt6)

#dt.2m<-dt.6.allsample.2m
#dt.1m<-dt.6.allsample.1m

#rm(dt.6.allsample.2m,dt.6.allsample.1m,dt.6.allsample)


c.pred.1m<-predict(model_LR.1m,dt.6.allsample.1m,type='response')
c.pred.1m<- ifelse( c.pred.1m > v1m,1,0)
sum(c.pred.1m)
mean(c.pred.1m)

c.pred.2m<-predict(model_LR.2m,dt.6.allsample.2m,type='response')
c.pred.2m <- ifelse( c.pred.2m > v2m,1,0)
sum(c.pred.2m)
mean(c.pred.2m)

prdct.1m<-as.data.table(
  data.frame(mobile=dt.6.allsample.1m$mobile,
             lost=c.pred.1m)
)
prdct.2m<-as.data.table(
  data.frame(mobile=dt.6.allsample.2m$mobile,
             lost=c.pred.2m)
)

prdct.1m$mobile<-as.double(prdct.1m$mobile)
prdct.2m$mobile<-as.double(prdct.2m$mobile)

write.csv(prdct.1m,"LOST_JNDS_201809_1M.CSV",row.names = F)
write.csv(prdct.2m,"LOST_JNDS_201809_2M.CSV",row.names = F)

#汇总入RFV
prdct<-rbind(prdct.1m,prdct.2m)
prdct<-prdct[!duplicated(prdct$mobile),]
length(unique(prdct$mobile))==nrow(prdct)
setnames(prdct,2,"pred201809")
dt.update<-fread("F:\\ZHZ_R\\RFVALL\\RFV整合201803-201808_181010updated.csv"
                 ,integer64 = 'numeric')
# dt.update<-dt.update[!duplicated(dt.update$mobile),]
# dt.update<-dt.update[,i.lost201808:=NULL]
length(unique(dt.update$mobile))==nrow(dt.update)

dt.update<-prdct[dt.update,on="mobile"]
dt.update[is.na(dt.update)]<-0
write.csv(dt.update,"F:\\ZHZ_R\\RFVALL\\RFV整合201803-201808_181010updated2.csv",row.names = F)


sum(dt.update$pred201809)






